{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Check Phrasal paper to see how they split the corpus\n",
    "# they only use en-fr from autodesk, not en-de"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import codecs\n",
    "import re\n",
    "from collections import Counter, OrderedDict, defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train_source_file = '/home/chris/projects/constrained_decoding/proto/autodesk_constrained_decoding_corpus/autodesk.train.en.bpe'\n",
    "# train_target_file = '/home/chris/projects/constrained_decoding/proto/autodesk_constrained_decoding_corpus/autodesk.train.de.bpe'\n",
    "\n",
    "source_tm_file = '/media/1tb_drive/parallel_data/wmt16_IT_domain/in_domain_train/indomain_tm/indomain.de-en.en.bpe'\n",
    "target_tm_file = '/media/1tb_drive/parallel_data/wmt16_IT_domain/in_domain_train/indomain_tm/indomain.de-en.de.bpe'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gazetteer_file = '/media/1tb_drive/parallel_data/wmt16_IT_domain/in_domain_train/wiki_all/gazetteerDE.tsv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from subword_nmt.apply_bpe import BPE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bpe_codes = open('/media/1tb_drive/parallel_data/en-de/chris_en-de_big_corpus/train/processed/all_text_both_EN_and_DE.79000.bpe.codes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "encoder = BPE(bpe_codes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from subprocess import Popen, PIPE\n",
    "\n",
    "# tokenization funcs\n",
    "SOURCE_LANG = 'en'\n",
    "TARGET_LANG = 'de'\n",
    "# TARGET_LANG = 'fr'\n",
    "\n",
    "\n",
    "tokenize_script = '/media/1tb_drive/parallel_data/en-de/chris_en-de_big_corpus/train/processed/tokenizer.perl'\n",
    "source_tokenizer_cmd = [tokenize_script, '-l', SOURCE_LANG, '-q', '-', '-no-escape', '1']\n",
    "target_tokenizer_cmd = [tokenize_script, '-l', TARGET_LANG, '-q', '-', '-no-escape', '1']\n",
    "\n",
    "# NOTE: it's much slower to create a new tokenizer every time, see entity linking for persistent tokenizer\n",
    "def tokenize(segment, src_trg):\n",
    "    if src_trg == 'source':\n",
    "        source_tokenizer = Popen(source_tokenizer_cmd, stdin=PIPE, stdout=PIPE)\n",
    "        segment, _ = source_tokenizer.communicate(segment.encode('utf-8'))\n",
    "    else:\n",
    "        target_tokenizer = Popen(target_tokenizer_cmd, stdin=PIPE, stdout=PIPE)\n",
    "        segment, _ = target_tokenizer.communicate(segment.encode('utf-8'))\n",
    "    \n",
    "    segment = segment.strip().decode('utf-8')\n",
    "    return segment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_gazetteer(filename):\n",
    "    with codecs.open(filename, encoding='utf8') as inp:\n",
    "        gazetteer_map = {}\n",
    "        \n",
    "        i = 0 \n",
    "        for line in inp:\n",
    "            deep, id_no, e, f = line.strip().split('\\t')\n",
    "            e = re.sub(u'\"', u'', e)\n",
    "            f = re.sub(u'\"', u'', f)\n",
    "            e = encoder.segment(tokenize(e, 'source'))\n",
    "            f = encoder.segment(tokenize(f, 'target'))\n",
    "            # TODO: tokenize first\n",
    "            if i % 50 == 0:\n",
    "                print(e)\n",
    "                print(f)\n",
    "            i += 1\n",
    "\n",
    "            gazetteer_map[e] = f\n",
    "    \n",
    "    return gazetteer_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# match English lowercase, map to foreign cased???? -- how much of a difference would this make?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "english\n",
      "title\n",
      "Nur@@ sing inform@@ atics\n",
      "Pfleg@@ ein@@ form@@ atik\n",
      "Analy@@ st\n",
      "Analy@@ st\n",
      "BL@@ AST\n",
      "BL@@ AS@@ T-@@ Algorithmus\n",
      "Computer-@@ supported cooperative work\n",
      "Computer Sup@@ ported Cooperative Work\n",
      "DNA bar@@ coding\n",
      "DNA-@@ Bar@@ coding\n",
      "European Robo@@ tics Platform\n",
      "EURO@@ P\n",
      "Geo@@ information\n",
      "Ge@@ od@@ aten\n",
      "Ho@@ dg@@ kin – Hu@@ x@@ ley model\n",
      "Ho@@ dg@@ k@@ in-@@ Hu@@ x@@ ley-@@ Modell\n",
      "IT@@ C En@@ sch@@ ede\n",
      "International Institute for Ge@@ o-@@ Information Science and Earth Observ@@ ation\n",
      "Dag@@ stuhl\n",
      "Leibni@@ z-@@ Zentrum f@@ ü@@ r Informatik\n",
      "Compar@@ ison of web application frameworks\n",
      "Liste von Web@@ frameworks\n",
      "Mixed reality\n",
      "Mixed reality\n",
      "Open@@ Ajax Alliance\n",
      "Open@@ Ajax Alliance\n",
      "Protein sub@@ cellular localization prediction\n",
      "Prot@@ ein@@ lok@@ alis@@ ations@@ vorher@@ sage\n",
      "Ru@@ le@@ ML Symposium\n",
      "Ru@@ le@@ ML Symposium\n",
      "Spe@@ ech recognition\n",
      "Spra@@ cher@@ kennung\n",
      "Toyota Partner Robot\n",
      "Toyota Partner Rob@@ ots\n",
      "Web mining\n",
      "Web Mining\n",
      "Adel@@ e Gold@@ berg ( computer scientist )\n",
      "Adel@@ e Gold@@ berg\n",
      "AN / U@@ Y@@ K-@@ 44\n",
      "AN / U@@ Y@@ K-@@ 44\n",
      "Ark@@ ady Vol@@ oz@@ h\n",
      "Ar@@ ka@@ di Jur@@ je@@ witsch W@@ olo@@ sch\n",
      "B@@ CH code\n",
      "B@@ CH@@ -Code\n",
      "Bot@@ ball\n",
      "Bot@@ ball\n",
      "Car@@ ver Me@@ ad\n",
      "Car@@ ver Me@@ ad\n",
      "Co@@ ding Technologies\n",
      "Co@@ ding Technologies\n",
      "Cu@@ th@@ bert Hurd\n",
      "Cu@@ th@@ bert Hurd\n",
      "Data transmission\n",
      "Daten@@ ü@@ ber@@ tragung\n",
      "Dennis Rit@@ chie\n",
      "Dennis Rit@@ chie\n",
      "Email encryption\n",
      "E-Mail-@@ Ver@@ schl@@ ü@@ ssel@@ ung\n",
      "Ta@@ il call\n",
      "Endre@@ kur@@ sion\n",
      "Fab@@ as@@ oft Foli@@ o\n",
      "Fab@@ as@@ oft Foli@@ o\n",
      "Fri@@ eder N@@ ake\n",
      "Fri@@ eder N@@ ake\n",
      "Geo@@ RSS\n",
      "Geo@@ RSS\n",
      "Google Maps\n",
      "Google Maps\n",
      "Heath@@ kit H@@ 8\n",
      "Heath@@ kit H@@ 8\n",
      "Ian Foster\n",
      "Ian Foster\n",
      "Inter@@ planetary Internet\n",
      "Inter@@ planet@@ ares Internet\n",
      "Java@@ Station\n",
      "Java@@ Station\n",
      "John P. Hay@@ es\n",
      "John P. Hay@@ es\n",
      "Ka@@ was@@ aki Heavy Industries\n",
      "Ka@@ was@@ aki Heavy Industries\n",
      "Alternative compensation system\n",
      "Kultur@@ flat@@ rate\n",
      "Linux Foundation\n",
      "Linux Foundation\n",
      "Low@@ est common ancest@@ or\n",
      "Low@@ est Common An@@ c@@ est@@ or\n",
      "Markov random field\n",
      "Markov Random Field\n",
      "Met@@ e@@ om@@ edia AG\n",
      "Met@@ e@@ om@@ edia\n",
      "Employ@@ ee monitoring\n",
      "Mitarbeiter@@ ü@@ ber@@ wach@@ ung\n",
      "Na@@ o ( robot )\n",
      "Na@@ o ( Roboter )\n",
      "OFF@@ IS\n",
      "OFF@@ IS – Institut f@@ ü@@ r Informatik\n",
      "O@@ ru@@ x@@ maps\n",
      "O@@ ru@@ x@@ Maps\n",
      "National identification number\n",
      "Person@@ enk@@ enn@@ zeichen\n",
      "Production planning\n",
      "Produktion@@ splanung und -@@ steuerung\n",
      "Raymond F. Boy@@ ce\n",
      "Raymond F. Boy@@ ce\n",
      "Robert C@@ ail@@ li@@ au\n",
      "Robert C@@ ail@@ li@@ au\n",
      "RO@@ V K@@ I@@ EL 6000\n",
      "RO@@ V K@@ I@@ EL 6000\n",
      "PV@@ CS\n",
      "Ser@@ ena PV@@ CS Version Manager\n",
      "Software@@ park Hag@@ enberg\n",
      "Software Competence Center Hag@@ enberg\n",
      "Steven Ru@@ dich\n",
      "Steven Ru@@ dich\n",
      "Tat@@ ort Internet\n",
      "Tat@@ ort Internet\n",
      "Tony Ho@@ are\n",
      "Tony Ho@@ are\n",
      "UN@@ IV@@ AC I\n",
      "UN@@ IV@@ AC I\n",
      "Vis@@ S@@ im\n",
      "Vis@@ S@@ im\n",
      "White label\n",
      "White label\n",
      "G@@ aj@@ ski-@@ Kuhn chart\n",
      "Y-@@ Diagramm\n",
      "4-@@ bit\n",
      "4@@ -Bit-@@ Architektur\n",
      "Active Template Library\n",
      "Active Template Library\n",
      "AES instruction set\n",
      "AES ( Befehls@@ sat@@ zer@@ weiterung )\n",
      "Ice Bu@@ cket Challenge\n",
      "ALS Ice Bu@@ cket Challenge\n",
      "Andrew Hu@@ x@@ ley\n",
      "Andrew Fiel@@ ding Hu@@ x@@ ley\n",
      "Apache Cam@@ el\n",
      "Apache Cam@@ el\n",
      "Motion ( software )\n",
      "Apple Motion\n",
      "Asphalt ( series )\n",
      "Asphalt ( Spiel@@ es@@ erie )\n",
      "A.@@ U.@@ G.@@ E.\n",
      "AU@@ GE\n",
      "Bark@@ er code\n",
      "Bark@@ er-@@ Code\n",
      "Be@@ OS\n",
      "Be@@ OS\n",
      "Bit bli@@ t\n",
      "Bit bli@@ t\n",
      "Blue Gene\n",
      "Blue Gene\n",
      "Brow@@ n-@@ bag seminar\n",
      "Brow@@ n@@ bag\n",
      "Cache ( computing )\n",
      "Cache\n",
      "CD@@ Bur@@ ner@@ XP\n",
      "CD@@ Bur@@ ner@@ XP\n",
      "Chip ( magazine )\n",
      "Chip ( Computer@@ zeitschrift )\n",
      "Click@@ j@@ acking\n",
      "Click@@ j@@ acking\n",
      "Colin Mc@@ Ra@@ e Rally\n",
      "Colin Mc@@ Ra@@ e Rally\n",
      "Num@@ er@@ ical control\n",
      "Computer@@ ized Num@@ er@@ ical Control\n",
      "Cour@@ sera\n",
      "Cour@@ sera\n",
      "CU@@ BR@@ ID\n",
      "CU@@ BR@@ ID\n",
      "Data General\n",
      "Data General\n",
      "Data entry cler@@ k\n",
      "Datent@@ yp@@ ist\n",
      "Dem@@ and@@ ware\n",
      "Dem@@ and@@ ware\n",
      "Digital An@@ vil\n",
      "Digital An@@ vil\n",
      "D@@ O-@@ 17@@ 8@@ B\n",
      "D@@ O-@@ 17@@ 8@@ B\n",
      "Dr. Dob@@ b 's Journal\n",
      "Dr. Dob@@ b ’ s Journal\n",
      "Dynamic programming\n",
      "Dynam@@ ische Programmierung\n",
      "1 % rule ( Internet culture )\n",
      "Ein-@@ Proz@@ ent-@@ Regel ( Internet )\n",
      "Enhanced Audio Codec\n",
      "Enhanced Audio Codec\n",
      "European Academic Research Network\n",
      "European Academic Research Network\n",
      "Fab@@ ian Th@@ yl@@ mann\n",
      "Fab@@ ian Th@@ yl@@ mann\n",
      "F@@ ID@@ O Alliance\n",
      "F@@ ID@@ O-@@ Allianz\n",
      "F@@ lock ( web browser )\n",
      "F@@ lock ( Browser )\n",
      "Free Pascal\n",
      "Free Pascal\n",
      "Fu@@ zzy classification\n",
      "Fu@@ zz@@ y-@@ Klassi@@ fikation\n",
      "G@@ Des@@ kl@@ ets\n",
      "G@@ Des@@ kl@@ ets\n",
      "Gib@@ bs ' inequality\n",
      "Gib@@ bs@@ -@@ Ungleich@@ ung\n",
      "GNU Lin@@ ear Programming Kit\n",
      "GNU Lin@@ ear Programming Kit\n",
      "Google Fi@@ ber\n",
      "Google Fi@@ ber\n",
      "Gre@@ ek@@ lish\n",
      "Gre@@ ek@@ lish\n",
      "H@@ al Fin@@ ney ( c@@ yp@@ her@@ punk )\n",
      "H@@ al Fin@@ ney\n",
      "Heroes of the Storm\n",
      "Heroes of the Storm\n",
      "Hop ( networking )\n",
      "Hop ( Netzwerk@@ technologie )\n",
      "IBM L@@ UM\n",
      "IBM L@@ UM\n",
      "In@@ compatible Tim@@ es@@ har@@ ing System\n",
      "In@@ compatible Tim@@ es@@ har@@ ing System\n",
      "Intel 80@@ 4@@ 86\n",
      "Intel 80@@ 4@@ 86\n",
      "Internet Research Steering Group\n",
      "Internet Research Steering Group\n",
      "ISO Master\n",
      "ISO Master\n",
      "Java User Group\n",
      "Java User Group\n",
      "Julian Se@@ ward\n",
      "Julian Se@@ ward\n",
      "K@@ Educ@@ a\n",
      "K@@ Educ@@ a\n",
      "Click fraud\n",
      "Kli@@ ck@@ betrug\n",
      "Kron@@ os Digital Entertainment\n",
      "Kron@@ os Digital Entertainment\n",
      "Large file support\n",
      "Large File Support\n",
      "Leonard Ad@@ leman\n",
      "Leonard Ad@@ leman\n",
      "Lin@@ ear complementarity problem\n",
      "Lin@@ ear@@ es Komplement@@ ar@@ it@@ ä@@ ts@@ problem\n",
      "Little CMS\n",
      "L@@ itt@@ le@@ CMS\n",
      "Loo@@ se coupling\n",
      "L@@ ose Kopplung\n",
      "Macro@@ management\n",
      "Macro@@ management\n",
      "Martin Hell@@ man\n",
      "Martin Hell@@ man\n",
      "Multi-@@ user\n",
      "Mehr@@ benutzer@@ system\n",
      "Micro@@ management ( gameplay )\n",
      "Micro@@ management\n",
      "Mike Mu@@ uss\n",
      "Mike Mu@@ uss\n",
      "Mobil@@ inux\n",
      "Mobil@@ inux\n",
      "Mozilla Europe\n",
      "Mozilla Europe\n",
      "Mun@@ in ( software )\n",
      "Mun@@ in ( Software )\n",
      "Nav@@ te@@ q\n",
      "Nav@@ te@@ q\n",
      "Network model\n",
      "Netzwerk@@ daten@@ bank@@ modell\n",
      "NO@@ ON state\n",
      "NO@@ ON-@@ Zustand\n",
      "O@@ dro@@ id\n",
      "OD@@ RO@@ ID\n",
      "Open Software License\n",
      "Open Software License\n",
      "IBM Tivoli Work@@ load Schedul@@ er\n",
      "Operation Planning and Control\n",
      "P@@ aid inclusion\n",
      "P@@ aid Inclu@@ sion\n",
      "Paul Se@@ ym@@ our ( mathemati@@ cian )\n",
      "Paul Se@@ ym@@ our\n",
      "Per@@ tec Computer\n",
      "Per@@ tec Computer Corporation\n",
      "Pla@@ y-@@ by-@@ mail game # Pla@@ y-@@ by-@@ Email\n",
      "Pla@@ y-@@ by-@@ E-Mail\n",
      "Power@@ set construction\n",
      "Pot@@ enz@@ mengen@@ konstruktion\n",
      "Program slic@@ ing\n",
      "Program S@@ lic@@ ing\n",
      "Public@@ .@@ Resour@@ c@@ e.@@ Or@@ g\n",
      "Public@@ .@@ resour@@ c@@ e.org\n",
      "Quart@@ z ( schedul@@ er )\n",
      "Quart@@ z ( Framework )\n",
      "Line drawing algorithm\n",
      "Rast@@ erung von Linien\n",
      "Red Star OS\n",
      "Red Star OS\n",
      "Res@@ et ( computing )\n",
      "Res@@ et\n",
      "Ro@@ gu@@ eli@@ ke\n",
      "Ro@@ gu@@ e-@@ like\n",
      "S@@ 3@@ M ( file format )\n",
      "S@@ 3@@ M\n",
      "Control panel ( engineering )\n",
      "Schal@@ tt@@ afel\n",
      "Se@@ af@@ ile\n",
      "Se@@ af@@ ile\n",
      "sequence number\n",
      "Sequ@@ enz@@ nummer\n",
      "SHO@@ UT@@ cast\n",
      "SHO@@ UT@@ cast\n",
      "Si@@ ri@@ us Software\n",
      "Si@@ ri@@ us Software\n",
      "Software as a service\n",
      "Software as a Service\n",
      "Spec@@ trum Holo@@ Byte\n",
      "Spec@@ trum Holo@@ Byte\n",
      "SQL Server Integration Services\n",
      "SQL Server Integration Services\n",
      "Exc@@ ess-@@ 3\n",
      "Sti@@ bit@@ z-@@ Code\n",
      "Su@@ dden Strike\n",
      "Su@@ dden Strike\n",
      "Syn@@ ap@@ tic ( software )\n",
      "Syn@@ ap@@ tic\n",
      "TA@@ O ( software )\n",
      "TA@@ O ( Software )\n",
      "Texas Instruments SN@@ 7@@ 64@@ 77\n",
      "Texas Instruments SN@@ 7@@ 64@@ 77\n",
      "Ti@@ le engine\n",
      "Ti@@ le-@@ Engine\n",
      "Track@@ M@@ ania 2 : Canyon\n",
      "Track@@ mania 2\n",
      "Tru@@ sted execution environment\n",
      "Tru@@ sted Exec@@ ution Environment\n",
      "Ultim@@ a ( series )\n",
      "Ultim@@ a ( Spiel@@ es@@ erie )\n",
      "Un@@ real@@ Script\n",
      "Un@@ real@@ Script\n",
      "Vi\n",
      "Vi\n",
      "Vor@@ bis\n",
      "Vor@@ bis\n",
      "Application firewall\n",
      "Web Application Firewall\n",
      "Windows DVD Maker\n",
      "Windows DVD Maker\n",
      "Wonder Boy\n",
      "Wonder Boy\n",
      "X@@ enix\n",
      "X@@ enix\n",
      "List of Gur@@ ren Lag@@ ann characters # Y@@ oko L@@ itt@@ ner\n",
      "Y@@ ō@@ ko R@@ itt@@ on@@ ā\n",
      ".NET Ref@@ lec@@ tor\n",
      ".NET Ref@@ lec@@ tor\n",
      "ACE ( compression file format )\n",
      "ACE ( Datei@@ format )\n",
      "Adobe Live@@ Cycle Reader Ex@@ tensions\n",
      "Adobe Live@@ Cycle Reader Ex@@ tensions ES\n",
      "Air@@ Drop\n",
      "Air@@ Drop\n",
      "X@@ ML@@ Sp@@ y\n",
      "Al@@ to@@ va X@@ ML@@ Sp@@ y\n",
      "An@@ vil ( game engine )\n",
      "An@@ vil ( Spiel@@ -@@ Engine )\n",
      "AR@@ IN@@ C 4@@ 24\n",
      "AR@@ IN@@ C 4@@ 24\n",
      "ATI Multi Ren@@ dering\n",
      "ATI Multi-@@ Ren@@ dering\n",
      "Back@@ door ( computing )\n",
      "Back@@ door\n",
      "Ex@@ amp@@ le.com\n",
      "Beispiel@@ domains\n",
      "Bio@@ Sho@@ ck Infin@@ ite\n",
      "Bio@@ Sho@@ ck Infin@@ ite\n",
      "Bo@@ eh@@ m garbage collector\n",
      "Bo@@ ehm@@ -@@ Speicher@@ ber@@ einigung\n",
      "B@@ read@@ crum@@ b ( navigation )\n",
      "Brot@@ kr@@ ü@@ mel@@ navigation\n",
      "Canter@@ bury corpus\n",
      "Canter@@ bury Cor@@ pus\n",
      "C@@ ev@@ at Y@@ er@@ li\n",
      "C@@ ev@@ at Y@@ er@@ li\n",
      "Cinema 4D\n",
      "Cinema 4D\n",
      "Code page 8@@ 57\n",
      "Co@@ dep@@ age 8@@ 57\n",
      "Common Data Format\n",
      "Common Data Format\n",
      "Connec@@ ted Limited Device Configuration\n",
      "Connec@@ ted Limited Device Configuration\n",
      "C@@ rank ( person )\n",
      "Cra@@ ck@@ pot\n",
      "CV@@ S@@ up\n",
      "CV@@ S@@ up\n",
      "Norwegian Data Protection Authority\n",
      "Dat@@ ati@@ l@@ sy@@ net ( Norwegen )\n",
      "De Blo@@ b\n",
      "De Blo@@ b\n",
      "De@@ S@@ mu@@ ME\n",
      "De@@ S@@ mu@@ ME\n",
      "Dig@@ raphs and tri@@ graphs\n",
      "Dig@@ raph ( Informatik )\n",
      "DivX\n",
      "DivX\n",
      "DOS Shell\n",
      "DOS Shell\n",
      "D@@ ump ( program )\n",
      "D@@ ump ( Software )\n",
      "Extension mechanisms for DNS\n",
      "ED@@ NS\n",
      "EM@@ T Ala@@ din\n",
      "EM@@ T Ala@@ din\n",
      "Extended Eu@@ cli@@ dean algorithm\n",
      "Erweiter@@ ter eu@@ kli@@ discher Algorithmus\n",
      "Extended Memory Speci@@ fication\n",
      "Extended Memory Speci@@ fication\n",
      "F@@ disk\n",
      "F@@ disk\n",
      "Fish F@@ ill@@ ets NG\n",
      "Fish F@@ ill@@ ets NG\n",
      "Fra@@ c@@ t@@ int\n",
      "Fra@@ c@@ t@@ int\n",
      "FT@@ L : F@@ aster Than Light\n",
      "FT@@ L : F@@ aster Than Light\n",
      "Gary Mc@@ K@@ inn@@ on\n",
      "Gary Mc@@ K@@ inn@@ on\n",
      "Qu@@ orum ( distributed computing ) # Qu@@ or@@ um-@@ based techniques in distributed database systems\n",
      "Gewi@@ cht@@ etes Vo@@ tieren\n",
      "Gold@@ Sr@@ c\n",
      "Gold@@ Sr@@ c\n",
      "Grand Th@@ eft Auto IV\n",
      "Grand Th@@ eft Auto IV\n",
      "Guadal@@ in@@ ex\n",
      "Guadal@@ in@@ ex\n",
      "Hal@@ stead complexity measures\n",
      "Hal@@ ste@@ ad-@@ Met@@ rik\n",
      "Hex@@ en : Beyond Her@@ etic\n",
      "Hex@@ en ( Computer@@ spiel )\n",
      "Honey@@ well R@@ Q-@@ 16 T-@@ Haw@@ k\n",
      "Honey@@ well R@@ Q-@@ 16\n",
      "Hyper@@ link\n",
      "Hyper@@ link\n",
      "Wireless Gig@@ abit Alliance\n",
      "IEEE 802.11@@ ad\n",
      "Information Presentation Facility\n",
      "Information Presentation Facility\n",
      "Internation@@ alized resource identifier\n",
      "Internation@@ alized Resource Identi@@ fier\n",
      "IS@@ CSI\n",
      "IS@@ CSI\n",
      "Java app@@ let\n",
      "Java-@@ App@@ let\n",
      "John D@@ rap@@ er\n",
      "John T. D@@ rap@@ er\n",
      "Kabel Deutschland\n",
      "Kabel Deutschland\n",
      "Kevin and K@@ ell\n",
      "Kevin & K@@ ell\n",
      "Seven Bri@@ dges of K@@ ö@@ ni@@ gs@@ berg\n",
      "K@@ ö@@ ni@@ gs@@ berger Br@@ ü@@ cken@@ problem\n",
      "Land@@ stal@@ ker : The Treas@@ ures of King N@@ ole\n",
      "Land@@ stal@@ ker\n",
      "Liberation fonts\n",
      "Liberation ( Schriftart )\n",
      "List of audio tra@@ ckers\n",
      "Liste von Track@@ ern\n",
      "Lotus 1-@@ 2-3\n",
      "Lotus 1-@@ 2-3\n",
      "Mac@@ Book\n",
      "Mac@@ Book\n",
      "Mail retrieval agent\n",
      "Mail Ret@@ ri@@ eval Agent\n",
      "Markus Per@@ sson\n",
      "Markus Per@@ sson\n",
      "Me@@ ebo\n",
      "Me@@ ebo\n",
      "Micro@@ -@@ Controller Operating Systems ( Micro@@ C / OS )\n",
      "Micro@@ C / OS-@@ II\n",
      "Mal@@ icious Software Rem@@ oval Tool\n",
      "Microsoft Tool zum Entfernen b@@ ö@@ s@@ artiger Software\n",
      "MIN@@ IX file system\n",
      "Mini@@ x ( Dateisystem )\n",
      "Moore machine\n",
      "Moor@@ e-@@ Autom@@ at\n",
      "M@@ ung ( computer term )\n",
      "M@@ ung@@ ing\n",
      "Nat@@ us Vinc@@ ere\n",
      "Nat@@ us Vinc@@ ere\n",
      "Net@@ H@@ ack\n",
      "Net@@ H@@ ack\n",
      "N@@ exu@@ iz\n",
      "N@@ exu@@ iz\n",
      "No@@ Script\n",
      "No@@ Script\n",
      "Oc@@ tree\n",
      "Oc@@ tree\n",
      "Open Software Foundation\n",
      "Open Software Foundation\n",
      "Portable Network Graphics # Optim@@ izing tools\n",
      "Opti@@ PNG\n",
      "Paper Mario\n",
      "Paper Mario\n",
      "PDF@@ Creator\n",
      "PDF@@ Creator\n",
      "Phot@@ om@@ ati@@ x Pro\n",
      "Phot@@ om@@ ati@@ x\n",
      "Pic@@ tive\n",
      "Plastic Interface for Collabor@@ ative Technology Initi@@ atives through Video Expl@@ oration\n",
      "Port scanner\n",
      "Por@@ ts@@ canner\n",
      "Professor Lay@@ ton and the Az@@ ran Legacy\n",
      "Professor Lay@@ ton und das Ver@@ m@@ ä@@ cht@@ nis von As@@ lant\n",
      "Bus bun@@ ching\n",
      "Pul@@ k@@ bildung\n",
      "Par@@ ticle statistics\n",
      "Quanten@@ statistik\n",
      "Rak@@ uten\n",
      "Rak@@ uten\n",
      "Re@@ dis\n",
      "Re@@ dis\n",
      "Resident Evil 2\n",
      "Resident Evil 2\n",
      "R@@ L ( complexity )\n",
      "R@@ L ( Komplex@@ it@@ ä@@ ts@@ klasse )\n",
      "RT@@ X : Real Time eX@@ tensions for Windows\n",
      "RT@@ X : Real Time eX@@ tensions f@@ ü@@ r Windows\n",
      "SAP Net@@ Weaver Process Integration\n",
      "SAP Process Integration\n",
      "Schnit@@ zel@@ mit@@ kar@@ toff@@ els@@ al@@ at\n",
      "Schnit@@ zel@@ mit@@ kar@@ toff@@ els@@ al@@ at\n",
      "Sender ID\n",
      "Sender ID\n",
      "S@@ him ( computing )\n",
      "S@@ him ( Informatik )\n",
      "Single-@@ page application\n",
      "Single-@@ pa@@ ge-@@ Web@@ anwendung\n",
      "Code smell\n",
      "Sm@@ ell ( Programmierung )\n",
      "Sonic & Seg@@ a All-@@ Stars Racing\n",
      "Sonic and Seg@@ a All-@@ Stars Racing\n",
      "Video game console\n",
      "Spiel@@ konsole\n",
      "Star@@ cross ( video game )\n",
      "Star@@ cross\n",
      "Stron@@ g@@ Swan\n",
      "Stron@@ g@@ Swan\n",
      "Super@@ Tux\n",
      "Super@@ Tux\n",
      "Tabu search\n",
      "Tab@@ u-@@ Suche\n",
      "Telef@@ ó@@ nica\n",
      "Telef@@ ó@@ nica\n",
      "The Butterfly Circus\n",
      "The Butterfly Circus ( Film )\n",
      "Time@@ Shift\n",
      "Tim@@ eshi@@ ft\n",
      "Tomb Rai@@ der : Under@@ world\n",
      "Tomb Rai@@ der : Under@@ world\n",
      "Transport Driver Interface\n",
      "Transport Driver Interface\n",
      "Intel Turbo Memory\n",
      "Turbo Memory\n",
      "Unique user\n",
      "Unique Visit\n",
      "US@@ U Software\n",
      "US@@ U Software\n",
      "Very High Speed ADSL\n",
      "Very High Speed ADSL\n",
      "Diagr@@ amming software\n",
      "Visu@@ alisierungs@@ programm\n",
      "Action Center\n",
      "Wartungs@@ center\n",
      "MSN TV\n",
      "Web@@ TV Networks\n",
      "William Crow@@ ther\n",
      "William Crow@@ ther\n",
      "Wing Commander Academy\n",
      "Wing Commander Academy ( Fernseh@@ serie )\n",
      "World Online\n",
      "World Online\n",
      "Xbox One\n",
      "Xbox One\n",
      "X@@ term\n",
      "X@@ term\n",
      "Z-@@ buff@@ ering\n",
      "Z-@@ Bu@@ ffer\n",
      "Access control\n",
      "Zugriffs@@ kontrolle\n",
      "1@@ C Company\n",
      "1@@ C ( Unternehmen )\n",
      "Abstra@@ ct data type\n",
      "Abstra@@ kter Datent@@ yp\n",
      "After@@ life ( video game )\n",
      "After@@ life ( Computer@@ spiel )\n",
      "IT@@ T System 12\n",
      "Alcat@@ el S@@ 12\n",
      "Am@@ ber@@ star\n",
      "Am@@ ber@@ star\n",
      "Anti@@ f@@ use\n",
      "Anti@@ fus@@ e-@@ Technologie\n",
      "AR@@ MA 2\n",
      "Arm@@ A 2\n",
      "A@@ stro Boy ( film )\n",
      "A@@ stro Boy – Der Film\n",
      "Auto@@ code\n",
      "Auto@@ code\n",
      "Bit@@ ap algorithm\n",
      "Ba@@ ez@@ a-@@ Y@@ ates-@@ Gon@@ net-@@ Algorithmus\n",
      "BC@@ PL\n",
      "Basic Combined Programming Language\n",
      "List of Bem@@ ani series\n",
      "Bem@@ ani\n",
      "Billy Mitchell ( video game player )\n",
      "Billy Mitchell ( Video@@ spieler )\n",
      "B@@ lek@@ ko\n",
      "B@@ lek@@ ko\n",
      "Born approximation\n",
      "Bor@@ n@@ sche N@@ ä@@ her@@ ung\n",
      "Bur@@ rows – Whe@@ eler transform\n",
      "Bur@@ row@@ s-@@ Whe@@ el@@ er-@@ Transformation\n",
      "Bul@@ ly ( video game )\n",
      "Can@@ is Can@@ em Edit\n",
      "Cha@@ Ch@@ a ( search engine )\n",
      "Cha@@ Ch@@ a\n",
      "Cit@@ adel / UX\n",
      "Cit@@ adel / UX\n",
      "Code generation\n",
      "Code@@ generi@@ erung\n",
      "Computer Oth@@ ello ( video game )\n",
      "Computer Oth@@ ello\n",
      "Cork encoding\n",
      "Cor@@ k-@@ Ko@@ dierung\n",
      "Cur@@ r@@ ent-@@ mode logic\n",
      "Current Mode Logic\n",
      "The House of Magic\n",
      "Das magische Haus\n",
      "Data de@@ duplication\n",
      "De@@ dup@@ likation\n",
      "Ring ( video game )\n",
      "Der Ring des Ni@@ bel@@ ungen ( Computer@@ spiel )\n",
      "The Madagascar Pen@@ gu@@ ins in a Christmas C@@ aper\n",
      "Die Madagas@@ car@@ -P@@ ingu@@ ine in vor@@ weih@@ nacht@@ licher Mission\n",
      "DirectX Video Acceler@@ ation\n",
      "DirectX Video Acceler@@ ation\n",
      "Don@@ key Kong Jungle Beat\n",
      "Don@@ key Kong Jungle Beat\n",
      "TO@@ CA Race Driver\n",
      "DTM Race Driver\n",
      "Eagle Dynamics\n",
      "Eagle Dynamics\n",
      "E@@ J@@ B Q@@ L\n",
      "E@@ J@@ B Q@@ L\n",
      "End@@ less Ocean\n",
      "End@@ less Ocean\n",
      "Ether@@ CAT\n",
      "Ether@@ CAT\n",
      "XHTML\n",
      "Exten@@ sible Hyper@@ text Mark@@ up Language\n",
      "Fast Local Internet Protocol\n",
      "Fast Local Internet Protocol\n",
      "Emergency Fire Response\n",
      "Fire Department ( Computer@@ spiel )\n",
      "Flying Wild Ho@@ g\n",
      "Flying Wild Ho@@ g\n",
      "Fre@@ ddy Ph@@ ark@@ as : Frontier Pharmac@@ ist\n",
      "Fre@@ ddy Ph@@ ark@@ as : Frontier Pharmac@@ ist\n",
      "Gal@@ aga\n",
      "Gal@@ aga\n",
      "G@@ ell-@@ Mann matri@@ ces\n",
      "G@@ ell-@@ Mann@@ -@@ Matri@@ zen\n",
      "G@@ mail\n",
      "G@@ mail\n",
      "Gran Tra@@ k 10\n",
      "Gran Tra@@ k 10\n",
      "Gy@@ rom@@ ite\n",
      "Gy@@ rom@@ ite\n",
      "Ed@@ na & Harvey : Harvey 's New E@@ yes\n",
      "Har@@ ve@@ ys neue Augen\n",
      "H@@ P-@@ IL\n",
      "Hew@@ lett@@ -@@ Packard Interface Loop\n",
      "HTC Smart\n",
      "HTC Smart\n",
      "IDS Sche@@ er\n",
      "IDS Sche@@ er\n",
      "IF@@ S AB\n",
      "Industrial and Financial Systems\n",
      "Inter@@ cep@@ tor pattern\n",
      "Inter@@ cep@@ tor ( Entwur@@ fs@@ muster )\n",
      "IPv6\n",
      "IPv6\n",
      "James Gun@@ n 's PG Por@@ n\n",
      "James Gun@@ n ’ s PG Por@@ n\n",
      "Je@@ wish@@ Gen\n",
      "Je@@ wish@@ Gen\n",
      "J@@ ump@@ gate : The Reconstruction Initiative\n",
      "J@@ ump@@ gate\n",
      "Key@@ word-@@ driven testing\n",
      "Key@@ word-@@ Dri@@ ven Testing\n",
      "Kno@@ l\n",
      "Kno@@ l\n",
      "K@@ Phone\n",
      "K@@ Phone\n",
      "LaserSoft Imaging\n",
      "LaserSoft Imaging\n",
      "Lib@@ v@@ n@@ c@@ server\n",
      "Lib@@ V@@ NC@@ Server\n",
      "List of IPv6 tunnel brokers\n",
      "Liste von IPv@@ 6@@ -T@@ unn@@ el@@ bro@@ kern\n",
      "The An@@ t Bul@@ ly ( film )\n",
      "Lucas , der Am@@ eisen@@ sch@@ reck\n",
      "Mag@@ ma ( company )\n",
      "MAG@@ MA\n",
      "Mas@@ on ( Perl )\n",
      "Mas@@ on ( Perl )\n",
      "MED@@ LINE\n",
      "MED@@ LINE\n",
      "Met@@ in@@ 2\n",
      "Met@@ in@@ 2\n",
      "MIL@@ -S@@ T@@ D-@@ 15@@ 53\n",
      "MIL@@ -S@@ T@@ D-@@ 15@@ 53\n",
      "Mo@@ j@@ oli@@ cious\n",
      "Mo@@ j@@ oli@@ cious\n",
      "Mr Hub@@ lot\n",
      "Mr Hub@@ lot\n",
      "Nam@@ co System 12\n",
      "Nam@@ co System 12\n",
      "Never Al@@ one ( video game )\n",
      "Never Al@@ one\n",
      "Nint@@ end@@ og@@ s + C@@ ats\n",
      "Nint@@ end@@ og@@ s + C@@ ats\n",
      "NX technology\n",
      "NX No@@ Machine\n",
      "One Man Band ( film )\n",
      "One Man Band\n",
      "Oracle Call Interface\n",
      "Oracle Call Interface\n",
      "Pac@@ -@@ Man and the G@@ ho@@ stly Adventures\n",
      "Pac@@ -@@ Man und die Geister@@ abent@@ euer\n",
      "Party@@ Gaming\n",
      "Party@@ Gaming\n",
      "Petro@@ gl@@ yp@@ h Games\n",
      "Petro@@ gl@@ yp@@ h Games\n",
      "Pir@@ an@@ ha Bytes\n",
      "Pir@@ an@@ ha Bytes\n",
      "Poly@@ bius square\n",
      "Poly@@ bi@@ os-@@ Chi@@ ff@@ re\n",
      "Pres@@ to ( layout engine )\n",
      "Pres@@ to ( Software )\n",
      "Protocol stack\n",
      "Protokoll@@ st@@ apel\n",
      "Quantum gravity\n",
      "Quanten@@ grav@@ itation\n",
      "Rainbow Islands : The Story of Bub@@ ble Bob@@ ble 2\n",
      "Rainbow Islands\n",
      "Real Time Streaming Protocol\n",
      "Real-Time Streaming Protocol\n",
      "Request Tracker\n",
      "Request Tracker\n",
      "RM@@ ON\n",
      "RM@@ ON\n",
      "H@@ ook flash\n",
      "R@@ ü@@ ck@@ fra@@ get@@ aste\n",
      "S@@ ame@@ Game\n",
      "S@@ ame@@ Game\n",
      "Ke@@ y-@@ agreement protocol\n",
      "Schl@@ ü@@ ssel@@ austausch@@ protokoll\n",
      "Self-@@ energy\n",
      "Selb@@ sten@@ ergie\n",
      "SH@@ A-@@ 3\n",
      "SH@@ A-@@ 3\n",
      "SIG@@ K@@ ILL\n",
      "SIG@@ K@@ ILL\n",
      "SIP Express Router\n",
      "SIP Express Router\n",
      "Sn@@ ap@@ chat\n",
      "Sn@@ ap@@ chat\n",
      "Soul Edge\n",
      "Soul Blade\n",
      "Spirit of Metal web@@ z@@ ine\n",
      "Spirit of Metal\n",
      "Star Trek : Klingon Academy\n",
      "Star Trek : Klingon Academy\n",
      "S@@ SE@@ 4\n",
      "Streaming SIM@@ D Ex@@ tensions 4\n",
      "Super PI\n",
      "Super PI\n",
      "System@@ atic code\n",
      "System@@ atischer Code\n",
      "Tec@@ mo\n",
      "Tec@@ mo\n",
      "Textile ( mark@@ up language )\n",
      "Textile\n",
      "Interactive Dis@@ assemb@@ ler\n",
      "The Interactive Dis@@ assemb@@ ler\n",
      "Adap@@ tations of The Wizard of O@@ z\n",
      "The Wizard of O@@ z ( Computer@@ spiel )\n",
      "Titanic : Adventure Out of Time\n",
      "Titanic : Adventure Out of Time\n",
      "Be@@ ast Wars : Transform@@ ers\n",
      "Transform@@ ers : Be@@ ast Wars\n",
      "Tur@@ tle ( syntax )\n",
      "Tur@@ tle ( Syntax )\n",
      "Un@@ char@@ ted 2 : Among Th@@ ieves\n",
      "Un@@ char@@ ted 2 : Among Th@@ ieves\n",
      "User experience\n",
      "User Experience\n",
      "Distri@@ buted computing\n",
      "Ver@@ teil@@ tes Rech@@ nen\n",
      "V@@ ME eX@@ tensions for Instru@@ mentation\n",
      "V@@ ME eX@@ tension for Instru@@ mentation\n",
      "War@@ hammer : Dark O@@ men\n",
      "War@@ hammer : Dark O@@ men\n",
      "W@@ et ( video game )\n",
      "W@@ et ( Computer@@ spiel )\n",
      "Windows Live\n",
      "Windows Live\n",
      "World Register of Marine Species\n",
      "World Register of Marine Species\n",
      "X@@ Chat\n",
      "X@@ Chat\n",
      "XT bus architecture\n",
      "X@@ T@@ -B@@ us-@@ Architektur\n",
      "Z@@ eng Jin@@ yan\n",
      "Z@@ eng Jin@@ yan\n",
      ".@@ ad\n",
      ".@@ ad\n",
      ".@@ cs\n",
      ".@@ cs\n",
      ".@@ ht\n",
      ".@@ ht\n",
      ".@@ mp\n",
      ".@@ mp\n",
      ".@@ sh\n",
      ".@@ sh\n",
      ".@@ y@@ t\n",
      ".@@ y@@ t\n",
      "Advanced Disc Fil@@ ing System\n",
      "Advanced Disc Fil@@ ing System\n",
      "An@@ j@@ uta\n",
      "An@@ j@@ uta\n",
      "AV@@ .@@ link\n",
      "A@@ V-@@ Link\n",
      "B@@ loo@@ d@@ line ( TV series )\n",
      "B@@ loo@@ d@@ line ( Fernseh@@ serie )\n",
      "C-@@ Netz\n",
      "C-@@ Netz\n",
      "Classi@@ fication Mark@@ up Language\n",
      "Classi@@ fication Mark@@ up Language\n",
      "Control@@ -@@ V\n",
      "Ctr@@ l-@@ V\n",
      "V@@ ESA Digital Flat Panel\n",
      "Digital Flat Panel\n",
      "Electro@@ -@@ optic effect\n",
      "Elektro@@ op@@ tischer Effekt\n",
      "Disk encryption\n",
      "Festplatten@@ ver@@ schl@@ ü@@ ssel@@ ung\n",
      "GB 180@@ 30\n",
      "GB 180@@ 30\n",
      "H@@ ax@@ e\n",
      "H@@ ax@@ e ( Programmiersprache )\n",
      "HTC One S\n",
      "HTC One S\n",
      "International Standard Name Identi@@ fier\n",
      "International Standard Name Identi@@ fier\n",
      "ISO 96@@ 60\n",
      "ISO 96@@ 60\n",
      "C@@ auc@@ asian K@@ not\n",
      "Ka@@ w@@ kas@@ ki U@@ sel\n",
      "RS@@ E Kri@@ ens ( Mis@@ sile )\n",
      "Lenk@@ waffe RS@@ E Kri@@ ens\n",
      "Lux@@ of@@ lux\n",
      "Lux@@ of@@ lux\n",
      "M@@ G@@ M-@@ 18 L@@ acro@@ sse\n",
      "M@@ G@@ M-@@ 18 L@@ acro@@ sse\n",
      "Motorola Back@@ flip\n",
      "Motorola Back@@ flip\n",
      "New@@ ton@@ Script\n",
      "New@@ ton@@ Script\n",
      "Oh@@ my@@ News\n",
      "Oh@@ my@@ News\n",
      "P@@ L-@@ 9\n",
      "P@@ L-@@ 9\n",
      "QU@@ IC\n",
      "Quick UDP Internet Connec@@ tions\n",
      "Ricoh\n",
      "Ricoh\n",
      "SA-@@ 10 G@@ rum@@ ble # Land@@ -based S-@@ 300@@ P .@@ 28@@ SA-@@ 10.@@ 29\n",
      "S-@@ 300@@ P\n",
      "Schneider Rundfunk@@ werke AG\n",
      "Schneider ( Unternehmen )\n",
      "Bor@@ land Side@@ kick\n",
      "Side@@ K@@ ick ( Software )\n",
      "Spe@@ ech Syn@@ thesis Mark@@ up Language\n",
      "Spe@@ ech Syn@@ thesis Mark@@ up Language\n",
      "St@@ oo@@ ge sort\n",
      "St@@ oo@@ ges@@ ort\n",
      "Telephone directory\n",
      "Telefon@@ buch\n",
      "T@@ I-@@ 83 series\n",
      "T@@ I-@@ 83\n",
      "Type 90 Shi@@ p-@@ to@@ -S@@ hip Mis@@ sile\n",
      "Typ@@ -@@ 80 AS@@ M-@@ 1\n",
      "Software version@@ ing\n",
      "Versions@@ nummer\n",
      "Wo@@ ol@@ ing\n",
      "Wo@@ ol@@ ing ( Magazin )\n",
      "State pattern\n",
      "Zustand ( Entwur@@ fs@@ muster )\n",
      "File attribute\n",
      "Datei@@ attri@@ but\n",
      "J ( programming language )\n",
      "J ( Programmiersprache )\n",
      "New Journal of Physics\n",
      "New Journal of Physics\n",
      "Schu@@ ko\n",
      "Schu@@ ko\n",
      "Aegean Numbers ( Unicode block )\n",
      "Uni@@ co@@ deb@@ lock Ä@@ g@@ ä@@ ische Zahl@@ zeichen\n",
      "Cha@@ m ( Unicode block )\n",
      "Uni@@ co@@ deb@@ lock Cha@@ m\n",
      "Combining Di@@ ac@@ ri@@ tical Marks Suppl@@ ement\n",
      "Uni@@ co@@ deb@@ lock Kombin@@ ierende di@@ ak@@ ri@@ tische Zeichen , Erg@@ ä@@ n@@ zung\n",
      "O@@ l Chi@@ ki ( Unicode block )\n",
      "Uni@@ co@@ deb@@ lock O@@ l Chi@@ ki\n",
      "En@@ closed Al@@ phan@@ umer@@ ics\n",
      "Uni@@ co@@ deb@@ lock Um@@ sch@@ lossene al@@ phan@@ um@@ erische Zeichen\n"
     ]
    }
   ],
   "source": [
    "gazet_map = parse_gazetteer(gazetteer_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# LOAD dev source, get all constraints, write json file with those constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_segments_from_file(filename):\n",
    "    with codecs.open(filename, encoding='utf8') as inp:\n",
    "        for i, l in enumerate(inp):\n",
    "            yield l.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "source_seg_iter = get_segments_from_file(source_tm_file)\n",
    "target_seg_iter = get_segments_from_file(target_tm_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tm_pair_map = {k:v for k,v in zip(source_seg_iter, target_seg_iter)\n",
    "                 if len(k) > 10 and len(k) < 50\n",
    "                 and len(v) > 10 and len(v) < 50\n",
    "                 and abs(len(k) - len(v)) < 15}\n",
    "tm_rules = [src for src in tm_pair_map.keys()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gazetteer_rules = [src for src,trg in gazet_map.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create terminology spotter\n",
    "from semantic_annotator.spotting import MatchSpotter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import the tokenized and bpe encoded dev lines\n",
    "# prepped_dev_lines = codecs.open('/home/chris/projects/constrained_decoding/proto/autodesk_constrained_decoding_corpus/autodesk.dev.1000.en.bpe',\n",
    "#                                  encoding='utf8').read().strip().split('\\n')\n",
    "\n",
    "# prepped_dev_lines = codecs.open('/home/chris/projects/constrained_decoding/proto/autodesk_constrained_decoding_corpus/en-fr/autodesk.dev.1000.en.bpe',\n",
    "#                                  encoding='utf8').read().strip().split('\\n')\n",
    "\n",
    "prepped_dev_lines = codecs.open('/media/1tb_drive/parallel_data/wmt16_IT_domain/in_domain_test/refs/en-de/Batch3a_en.bpe',\n",
    "                                 encoding='utf8').read().strip().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for k,v in tm_pair_map.items()[:100]:\n",
    "#     print(u'key: {} val: {}'.format(k,v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\" Computer@@ ized maintenance management system \"\n",
      "\" Computer@@ ized Maintenance Management System \"\n"
     ]
    }
   ],
   "source": [
    "keys_by_len = sorted(tm_rules, key=lambda x: len(x))\n",
    "print(keys_by_len[-100])\n",
    "print(tm_pair_map[keys_by_len[-100]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "270\n"
     ]
    }
   ],
   "source": [
    "tm_spotter = MatchSpotter(rules=tm_rules)\n",
    "\n",
    "# num constraints from TM\n",
    "dev_term_spots = []\n",
    "for l in prepped_dev_lines:\n",
    "    spots = tm_spotter.get_spots(l)\n",
    "    dev_term_spots.append(spots)\n",
    "\n",
    "dev_term_constraints = []\n",
    "for text, spots in zip(prepped_dev_lines, dev_term_spots):\n",
    "    output_constraints = []\n",
    "    if len(spots) > 0:\n",
    "        for spot in spots:\n",
    "            if spot[1] - spot[0] > 1:\n",
    "                spotted_term = text[spot[0]:spot[1]]\n",
    "#                 print(spotted_term)\n",
    "                mapped_term = tm_pair_map[spotted_term]\n",
    "                output_constraints.append(mapped_term.split())\n",
    "    dev_term_constraints.append(output_constraints)\n",
    "    \n",
    "print(len([c for l in dev_term_constraints for c in l]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "370\n"
     ]
    }
   ],
   "source": [
    "# num constraints from Gazetteer\n",
    "gazetteer_spotter = MatchSpotter(rules=gazetteer_rules)\n",
    "\n",
    "# num constraints from TM\n",
    "dev_term_spots = []\n",
    "for l in prepped_dev_lines:\n",
    "    spots = gazetteer_spotter.get_spots(l)\n",
    "    dev_term_spots.append(spots)\n",
    "\n",
    "dev_term_constraints = []\n",
    "for text, spots in zip(prepped_dev_lines, dev_term_spots):\n",
    "    output_constraints = []\n",
    "    if len(spots) > 0:\n",
    "        for spot in spots:\n",
    "            if spot[1] - spot[0] > 1:\n",
    "                spotted_term = text[spot[0]:spot[1]]\n",
    "#                 print(spotted_term)\n",
    "                mapped_term = gazet_map[spotted_term]\n",
    "                output_constraints.append(mapped_term.split())\n",
    "    dev_term_constraints.append(output_constraints)\n",
    "    \n",
    "print(len([c for l in dev_term_constraints for c in l]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# combine both maps\n",
    "combined_map = tm_pair_map.copy()\n",
    "combined_map.update(gazet_map) # which returns None since it mutates combined_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "578\n"
     ]
    }
   ],
   "source": [
    "# num constraints from Gazetteer\n",
    "combined_spotter = MatchSpotter(rules=[k for k,v in combined_map.items()])\n",
    "\n",
    "# num constraints from TM\n",
    "dev_term_spots = []\n",
    "for l in prepped_dev_lines:\n",
    "    spots = combined_spotter.get_spots(l)\n",
    "    dev_term_spots.append(spots)\n",
    "\n",
    "dev_term_constraints = []\n",
    "for text, spots in zip(prepped_dev_lines, dev_term_spots):\n",
    "    output_constraints = []\n",
    "    if len(spots) > 0:\n",
    "        for spot in spots:\n",
    "            if spot[1] - spot[0] > 1:\n",
    "                spotted_term = text[spot[0]:spot[1]]\n",
    "#                 print(spotted_term)\n",
    "                mapped_term = combined_map[spotted_term]\n",
    "                output_constraints.append(mapped_term.split())\n",
    "    dev_term_constraints.append(output_constraints)\n",
    "    \n",
    "print(len([c for l in dev_term_constraints for c in l]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dev_term_constraints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "# OUTPUT_DIR='/home/chris/projects/constrained_decoding/proto/autodesk_constrained_decoding_corpus'\n",
    "# OUTPUT_DIR='/home/chris/projects/constrained_decoding/proto/autodesk_constrained_decoding_corpus/en-fr'\n",
    "OUTPUT_DIR='/media/1tb_drive/parallel_data/wmt16_IT_domain/in_domain_test/refs/en-de/'\n",
    "\n",
    "with codecs.open(os.path.join(OUTPUT_DIR, 'tm_and_gazet.test.constraints.json'), 'w', encoding='utf8') as out:\n",
    "    out.write(json.dumps(dev_term_constraints, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
